import requests
import re
from collections import OrderedDict
import hashlib
import os
import json
import pymysql
import time
# from PIL import Image


class Mysql():
    def __init__(self,db):
        self.conn = pymysql.connect(host='127.0.0.1', port=3306, user='root', db=db, passwd='123456',
                                    charset='utf8')
        self.csl = self.conn.cursor()
        self.dictCsl = self.conn.cursor(pymysql.cursors.DictCursor)

    def commit(self):
        self.conn.commit()

    def insert_sql(self,table,data,db=None):
        fields = '`' + '`,`'.join(data.keys()) + '`'
        values = []
        for v in data.values():
            if v == 0:
                v = '0'
            elif not v:
                v = ''
            if type(v) == int :
                values.append(v.__str__())
            elif v == "now()":
                values.append(v)
            else:
                values.append("'%s'" % v.replace("'", " ").replace("\\", "\\\\"))
        if db:
            sql = 'INSERT INTO `%s`.`%s` (%s) VALUES (%s)' % (db, table, fields, ",".join(values))
        else:
            sql = 'INSERT INTO `%s` (%s) VALUES (%s)' % (table, fields, ",".join(values))
        return sql


class Hyundai():
    modis_list = ['MO', 'MI', 'CH', 'BO', 'TR', 'EL']
    modis_dict = {'MO': [1, '发动机'], 'MI': [2, '变速器'], 'CH': [3, '底盘'], 'BO': [4, '车身'], 'TR': [5, '内饰'],
                  'EL': [6, '电器']}
    def __init__(self,brand,regn):
        self.headers = {
            'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.110 Safari/537.36',
            'Cookie':'JSESSIONID=0000PswAoVbkEB9DcJZutYeBhvi:1b9i8aoq5; WPCLOGIN=PswAoVbkEB9DcJZutYeBhvi',
            'Host':'wpc.mobis.co.kr',
            'Key':'ef8bf19f-7b53-444c-bb75-9451c95448ca',
            'ClientSign':'d1287c3bc062016260ffc859e8c3f662',
            'Referer':'http://wpc.mobis.co.kr/Index/Index.jsp',
            'Accept':'*/*',
            'Accept-Language':'zh-CN,zh;q=0.8;q=0.8',
        }
        self.regn = regn
        self.brandCode = brand
        if brand == 'kia':
            self.hkgb = 'K'
        elif brand == 'hyundai':
            self.hkgb = 'H'
        # 类型url
        self.nameUrl = 'http://wpc.mobis.co.kr/Model?cmd=getCatalogSegmentList&hkgb=' + self.hkgb + '&regn=' + self.regn + '&views=Y&sgmts=C%2CE%2CP%2CR%2CS&bpnos='
        # 车型url
        self.codeUrl = 'http://wpc.mobis.co.kr/Model?cmd=getCatalogList&hkgb=' + self.hkgb + '&regn=' + self.regn + '&views=Y&sgmts=C%2CE%2CP%2CR%2CS&bpnos=&sgmt={}'
        # 车型信息url
        self.carUrl = 'http://wpc.mobis.co.kr/Model?cmd=getCatalogInfo&bpno={}&hsgb=NEW'
        self.modisUrl = 'http://wpc.mobis.co.kr/IllustSearch?cmd=getUniqueGroupKeyList'
        self.Gurl = 'http://wpc.mobis.co.kr/IllustSearch?cmd=getFinalGroupKeyList&table=CDM&group={}&match=ON&isVin=MODEL'
        self.detailUrl = 'http://wpc.mobis.co.kr/TextSearch?cmd=getFinalGroupDataList&table=CDM&group={}&pncd={}&match=ON&isVin=MODEL&mf=A%2C'
        self.psort = 0
        self.count = 0
        self.sqlCount = 0
        self.regnDict = {'CHN':'国内','GEN':'国外'}
        self.timeout = 20

    def get_response(self,url,img=None):
        # 发起请求
        if img:
            response = requests.get(url,headers=self.headers,proxies={'http':'101.132.46.34:5870'}).content
            number = 0
            while True:
                number += 1
                if b'<!DOCTYPE' in response and number <= 2:
                    response = requests.get(url, headers=self.headers, timeout=10, proxies={'http': '101.132.46.34:5870'}).content
                elif b'<!DOCTYPE' in response and number <= 4:
                    url = re.sub('TIF','PNG',url)
                    response = requests.get(url, headers=self.headers, timeout=10, proxies={'http': '101.132.46.34:5870'}).content
                else:
                    break
        else:
            response = requests.get(url,headers=self.headers, timeout=10, proxies={'http':'101.132.46.34:5870'}).content.decode('utf-8')
        print('发起请求')
        return response

    def parseName(self,response):
        nameList = response.split('\\@')
        nameDict = {}
        for name_ in nameList:
            if not name_:
                continue
            name = name_.split('\\:')[0]
            nameDict[name] = name_.split('\\:')[1]
        return nameDict

    def parse_response(self,response):
        # 处理响应内容，准备构建POST参数
        cars_list = response.split('\\@')
        # print(cars_list)
        return cars_list

    def parse_postdata(self,postdata_list):
        # 构建POST参数
        data = OrderedDict()
        data['CRNM'] = postdata_list[0]
        data['BPNO'] = postdata_list[1]
        data['HKGB'] = postdata_list[2]
        data['HSGB'] = postdata_list[3]
        data['OPTY'] = postdata_list[4]
        data['PCCD'] = postdata_list[5]
        data['PKGB'] = postdata_list[6]
        data['VHCD'] = postdata_list[7]
        data['VIEW'] = postdata_list[8]
        data['EFIN'] = postdata_list[9]
        data['EFOT'] = postdata_list[10]
        data['UCCT'] = postdata_list[-1]
        data['SGMT'] = postdata_list[-2]
        data['GRTY'] = postdata_list[-3]
        data['ISVIN'] = 'MODEL'
        data['LANG'] = 'CH'
        data['LRCD'] = 'EN'
        data['MATCH'] = 'ON'
        data['AS'] = 'N'
        return data

    def setPostdata(self,postdata):
        # 构建POST参数
        data = OrderedDict()
        data['CRNM'] = postdata['CRNM']
        data['BPNO'] = postdata['BPNO']
        data['HKGB'] = postdata['HKGB']
        data['HSGB'] = postdata['HSGB']
        data['OPTY'] = postdata['OPTY']
        data['PCCD'] = postdata['PCCD']
        data['PKGB'] = postdata['PKGB']
        data['VHCD'] = postdata['VHCD']
        data['VIEW'] = postdata['VIEW']
        data['EFIN'] = postdata['EFIN']
        data['EFOT'] = postdata['EFOT']
        data['UCCT'] = postdata['UCCT']
        data['SGMT'] = postdata['SGMT']
        data['GRTY'] = postdata['GRTY']
        data['ISVIN'] = 'MODEL'
        data['LANG'] = 'CH'
        data['LRCD'] = postdata['LRCD']
        data['MATCH'] = 'ON'
        data['AS'] = postdata['AS']
        data['GRNO'] = postdata['GRNO']
        return data

    def parse_cars(self,data):
        cars = data.split('\\:')
        return cars

    def parse_data(self,response):
        data = response.split('\\@')[0]
        data_list = data.split('\\:')
        return data_list

    def read_response(self,html_name,HKGB='K',dirs=None):
        if not dirs:
            file_path = '/Users/a058/Desktop/hyundai_' + HKGB + '/' + html_name + '.html'
        else:
            file_path = '/Users/a058/Desktop/hyundai_' + HKGB + '/'+ dirs + '/' + html_name + '.html'
        if os.path.exists(file_path):
            with open(file_path, 'r')as f:
                response = f.read()
                print('正在读取')
        else:
            response = None
        return response

    def write_response(self, html_name, response, HKGB='K',dirs=None,img=None):
        if dirs:
            path = '/Users/duanmengdi/Desktop/workspace/py3_code/hk/' + HKGB + '/' + dirs
            if not os.path.exists(path):
                os.makedirs(path)
            if img:
                with open('/Users/duanmengdi/Desktop/workspace/py3_code/hk/' + HKGB + '/' + dirs + '/' + html_name + '.png', 'wb')as f:
                    f.write(response)
                    print('正在写入图片'+ dirs + '/' +  html_name)
            else:
                with open(path + '/' + html_name + '.html', 'w')as f:
                    f.write(response)
                    print('正在写入')
        else:
            with open('/Users/duanmengdi/Desktop/workspace/py3_code/hk/' + HKGB + '/' + html_name + '.html', 'w')as f:
                f.write(response)
                print('正在写入')

    def read_post_response(self,html_name,post_url,data, HKGB='K',dirs=None):
        # 获取post请求响应内容
        response = self.read_response(html_name,HKGB=HKGB,dirs=dirs)
        if response:
            if 'ReadResponse' in response or 'Exception' in response:
                response = None
        if not response:
            print('本地暂无')
            response = requests.post(url=post_url, data=data, headers=self.headers,proxies={'http': '101.132.46.34:5870'}).content.decode()
            while 'ReadResponse' in response or 'Exception' in response:
                response = requests.post(url=post_url, data=data, headers=self.headers,
                                         proxies={'http': '101.132.46.34:5870'}).content.decode()
            # self.write_response(html_name, response, HKGB=HKGB,dirs=dirs)
        return response

    def read_get_response(self,url,HKGB='K',dirs=None):
        # 获取get请求响应内容
        html_name = hashlib.md5(url.encode()).hexdigest()
        response = self.read_response(html_name,HKGB=HKGB,dirs=dirs)
        # print(response)
        if not response:
            print('本地暂无')
            response = self.get_response(url)
            # 发起请求，获取响应

            # self.write_response(html_name, response,HKGB=HKGB,dirs=dirs)
            # 写入文件保存
        return response

    def parse_fenzu(self,txt):
        # 提取出分组数据
        if txt.endswith('@'):
            txt = txt.strip('@')
        try:
            new_list = re.split(r'@',txt)[1]
        except Exception:
            new_list = re.split(r'@', txt)[0]
        print(new_list)
        new_list = new_list.split('\^')
        return new_list

    def zizu_parse_img(self,zizu_img):
        img_list = re.split(r'\^',zizu_img)
        item_list = []
        # print(img_list)
        for i in img_list:
            num_list = re.split(r'\\:',i)
            # print(num_list)
            if num_list == ['']:
                continue
            item = dict()
            # print(num_list)
            fac = 0.68
            item['hsKey'] = num_list[3]
            item['page'] = num_list[1]
            item['hsX'] = int(int(num_list[4]) * fac)
            item['hsY'] = int(int(num_list[5]) * fac)
            item['maxX'] = int(item['hsX'] + int(num_list[6]) * fac)
            item['maxY'] = int(item['hsY'] + int(num_list[7]) * fac)
            # print(item)
            item_list.append(item)
            # print(item_dict_)
        return item_list

    def parse_item_list(self,item_list):
        item_set = set()
        for i in item_list:
            item_set.add(i['page'])
        item = dict()
        for m in item_set:
            list_1 = []
            for i in item_list:
                if i['page'] == m:
                    list_1.append(i)
            item[m] = list_1
        for i in item_list:
            i.pop('page')
        return item

    def parse_item_list_(self,item_list):
        item_set = set()
        for i in item_list:
            item_set.add(i['page'])
        item = dict()
        for m in item_set:
            dict_1 = dict()
            for i in item_list:
                if i['page'] == m:
                    dict_1[i['hsKey']] = i
            item[m] = dict_1
        return item

    def perse_fenzu_img(self,response):
        data_list = response.split('\@')[0]
        num_set = set()
        test_list = re.split(r'\^',data_list)
        for test in test_list:
            data = re.split(r'\\:',test)[1]
            if data == '':
                continue
            num_set.add(data)

        return num_set

    def read_img(self,dirs,html_name,HKGB='K'):
        file_path = '/Users/a058/Desktop/hyundai_' + HKGB + '/' + dirs + '/' + html_name + '.png'
        with open(file_path,'rb')as f:
            response = f.read()
        print(file_path)
        return response

    def toPng(self,dirs,img_name,im_name,HKGB='K',result=None):
        file_path = '/Users/duanmengdi/Desktop/workspace/py3_code/hk/' + HKGB + '/' + dirs + '/' + img_name + '.png'
        im = Image.open(file_path)
        if not result:
            try:
                im.save('/Users/a058/Desktop/' + self.brandCode + '_img' + '/' + dirs + '/'+ im_name + '.png','png')
            except Exception as e:
                os.makedirs('/Users/a058/Desktop/' + self.brandCode + '_img' + '/' + dirs)
                im.save('/Users/a058/Desktop/' + self.brandCode + '_img' + '/' + dirs + '/'+ im_name + '.png','png')
        imageHeigh = im.height
        imageWidth = im.width
        return imageHeigh, imageWidth

    def carsInfo_sql(self, code, car_name, name_dict, i_name, new_mysql):
        sql_data = dict()
        sql_data['brand'] = self.brandCode
        sql_data['brandCode'] = self.brandCode
        sql_data['code'] = code
        sql_data['uid'] = hashlib.md5((car_name[0] + car_name[1]).encode()).hexdigest()
        sql_data['condition'] = name_dict[i_name]
        sql_data['name'] = self.regnDict[self.regn]
        sql_data['status'] = 2
        sql_data['createtime'] = "now()"
        sql_data['updatetime'] = "now()"
        sql_data['extra_info'] = ""
        sql_data['limit'] = car_name[0]
        sql_data['limit2'] = ""
        sql_data['limit3'] = ""
        sql_data['limit4'] = ""
        sql_data['label'] = ""
        sql_data['description'] = ""
        sql_data['imagePath'] = ""
        sql_data['imgName'] = ""
        sql = new_mysql.insert_sql('cars_info_collection_add',
                                   sql_data) + 'ON DUPLICATE KEY UPDATE `label`=values(`label`),`imgName`=values(`imgName`),`imagePath`=values(`imagePath`),`description`=values(`description`),`limit2`=values(`limit2`),`limit3`=values(`limit3`),`limit4`=values(`limit4`),`limit`=values(`limit`),`brandCode`=values(`brandCode`),`name`=values(`name`),`brand`=values(`brand`),`code`=values(`code`),`condition`=values(`condition`),`extra_info`=values(`extra_info`),`createtime`=values(`createtime`),`updatetime`=values(`updatetime`) '
        return sql

    def group_collection_sql(self, data, mod, new_mysql):
        sql_data = dict()
        sql_data['num'] = self.modis_dict[mod][0]
        sql_data['label'] = self.modis_dict[mod][1]
        sql_data['cid'] = data['BPNO']
        sql_data['brandCode'] = self.brandCode
        sql_data['uid'] = hashlib.md5((sql_data['cid'] + mod).encode()).hexdigest()
        sql_data['createtime'] = "now()"
        sql_data['updatetime'] = "now()"
        sql_data['extra_info'] = mod
        sql = new_mysql.insert_sql('cars_group_collection_add',
                                   sql_data) + 'ON DUPLICATE KEY UPDATE `brandCode`=values(`brandCode`),`cid`=values(`cid`),`num`=values(`num`),`label`=values(`label`),`extra_info`=values(`extra_info`),`createtime`=values(`createtime`),`updatetime`=values(`updatetime`) '
        new_mysql.csl.execute(sql)
        return sql_data

    def structure_sql(self, page_item, img, im_hight, im_width, html_name, data, im_uid):
        imgPoints = dict()
        imgPoints["hotspots"] = page_item[img[1]]
        imgPoints["imageFormat"] = 'image/png'
        imgPoints['maxScaleFactor'] = 1.0
        imgPoints["imageHeight"] = im_hight
        imgPoints["imageWidth"] = im_width
        imgPoints = json.dumps(imgPoints)
        # 构建sql语句
        sql_data = dict()
        sql_data['uid'] = html_name + img[1]
        sql_data['cid'] = data['BPNO']
        sql_data['num'] = self.modis_dict[data['LRCD']][0]
        sql_data['maingroup'] = sql_data['num']
        sql_data['mid'] = re.sub('-', '', data['GRNO']) + img[1]
        sql_data['imgPoints'] = imgPoints
        sql_data['subgroup'] = sql_data['mid'][:2]
        sql_data['name'] = data['strucName']
        sql_data['brandCode'] = self.brandCode
        sql_data['createtime'] = "now()"
        sql_data['updatetime'] = "now()"
        sql_data['imageLarge'] = im_uid
        sql_data['extra_info'] = ""
        sql_data['condition'] = ""
        sql_data['model'] = ""
        sql_data['description'] = ""
        sql_data['logo'] = ""
        sql_data['imgName'] = ""
        return sql_data, imgPoints

    def partsPrice_sql(self, detail_data):
        sql_data = dict()
        sql_data['brandCode'] = self.brandCode
        sql_data['name'] = '4S'
        sql_data['pid'] = detail_data[2]
        sql_data['channel'] = '000'
        sql_data['retailprice'] = detail_data[7]
        sql_data['label'] = detail_data[3]
        sql_data['qty'] = detail_data[6]
        sql_data['status'] = 1
        sql_data['createtime'] = "now()"
        sql_data['condition'] = ""
        return sql_data

    def carsPartNum_sql(self, w, data, detail_data):
        sql_data = OrderedDict()
        extra_info = OrderedDict()
        try:
            extra_info['date'] = detail_data[0] + '-' + detail_data[1]
        except:
            extra_info['date'] = ''
        uc_ = re.findall(r'\[([^]]*)\]', detail_data[13])
        ucStr = detail_data[13].split('[')[0]
        if ucStr and ucStr != ' ':
            uc = re.sub(r' ', '', ucStr)
            if uc == 'EV1' or uc == 'EV2':
                extra_info['ev'] = [uc]
            else:
                extra_info['uc'] = re.split(r'>', uc)
        if uc_:
            extra_info['sa'] = uc_[0].split(', ')
        extra_info['date'] = detail_data[0] + "-" + detail_data[1]
        sql_data['cid'] = data['BPNO']
        sql_data['num'] = self.modis_dict[data['LRCD']][0]
        sql_data['maingroup'] = sql_data['num']
        sql_data['mid'] = re.sub('-', '', data['GRNO']) + w
        sql_data['subgroup'] = sql_data['mid'][:2]
        sql_data['brandCode'] = self.brandCode
        sql_data['pnum'] = data['PNCD']
        sql_data['itid'] = data['PNCD']
        sql_data['pid'] = detail_data[2]
        sql_data['label'] = detail_data[3]
        sql_data['quantity'] = detail_data[6]
        sql_data['price'] = detail_data[7]
        remark = re.sub('\[', '', detail_data[4])
        sql_data['remark'] = re.sub('\]', '', remark)
        sql_data['createtime'] = "now()"
        sql_data['updatetime'] = "now()"
        sql_data['extra_info'] = json.dumps(extra_info)
        sql_data['model'] = ""
        sql_data['rid'] = ""
        sql_data['img'] = ""
        sql_data['logo'] = ""
        sql_data['condition'] = ""
        sql_data['replacelabel'] = ""
        sql_data['isreplace'] = ""
        self.psort += 1
        return sql_data

    def image_sql(self, im_uid, imgPoints, data, im_name):
        sql_data = dict()
        sql_data['uid'] = im_uid
        sql_data['imgPoints'] = imgPoints
        sql_data['brandCode'] = self.brandCode
        sql_data['createtime'] = "now()"
        sql_data['updatetime'] = "now()"
        sql_data['status'] = 1
        sql_data['imagePath'] = data['BPNO'] + '/' + im_name
        return sql_data

    def postData_sql(self,new_mysql, uid):
        self.PostData['uid'] = uid
        sql = new_mysql.insert_sql('post_data_' + self.regn,
                                   self.PostData)
        new_mysql.csl.execute(sql)

    def GpostData_sql(self,new_mysql):
        try:
            self.PostData['uid'] = self.PostData['BPNO'] + self.PostData['LRCD'] +self.PostData['GRNO']
            sql = new_mysql.insert_sql('Gpost_data_' + self.regn, self.PostData) + 'ON DUPLICATE KEY UPDATE `BPNO`=values(`BPNO`)'
            new_mysql.csl.execute(sql)
        except:
            with open('Gpost_data','a')as f:
                f.write(self.PostData['uid'])

    def strucImg(self,page_set,data,html_name):
        image_dict = dict()
        for page in page_set:
            img_url_1 = 'http://wpc.mobis.co.kr/illust/LGROUPM/GIF/' + data['GRTY'] + '_' + data[
                'LRCD'] + page + '.gif?time=' + str(int(time.time()))
            img_url_1 = re.sub(r'\*', '%5E', img_url_1)
            # img_response = self.get_response(img_url_1, img=True)
            img_name = data['GRTY'] + '_' + data['LRCD'] + page
            try:
                img_response = self.read_img(data['BPNO'], img_name, HKGB=data['HKGB'])
            except FileNotFoundError:
                img_response = self.get_response(img_url_1, img=True)
                self.write_response(img_name, img_response, HKGB=data['HKGB'], dirs=data['BPNO'], img=True)
            im_name = data['BPNO'] + data['LRCD'] + page
            if b'<!DOCTYPE' in img_response:
                img_url_1 = re.sub(r'\*', '%5E', img_url_1)
                img_response = self.get_response(img_url_1, img=True)
                self.write_response(img_name, img_response, HKGB=data['HKGB'], dirs=data['BPNO'], img=True)
                print(img_response)
            try:
                im_name = self.toPng(data['BPNO'], img_name, im_name=im_name, HKGB=data['HKGB'])
                image_dict[page] = im_name
            except Exception as e:
                print(e)
                print('图片错误')
            # self.write_response(img_name, img_response, dirs=data['BPNO'], img=True)
        return image_dict

    def StrucData(self,new_mysql):
        sql = new_mysql.insert_sql('StrucData_' + self.regn,
                                   self.PostData) + 'ON DUPLICATE KEY UPDATE `zizu_img_n`=values(`zizu_img_n`), `new_zizu`=values(`new_zizu`), `page_item_`=values(`page_item_`), `page_item`=values(`page_item`)'
        new_mysql.csl.execute(sql)

    def stoCarInfo(self,new_mysql, old_mysql):
        nameResponse = self.read_get_response(self.nameUrl)
        # 汽车类型字典
        name_dict = self.parseName(nameResponse)
        for i_name in name_dict.keys():
            response = self.read_get_response(self.codeUrl.format(i_name))
            # print(response)
            cars_list = self.parse_response(response)
            for cars in cars_list:
                car_name = self.parse_cars(cars)
                if car_name == ['']:
                    continue
                sql = '''select * from cars_info_collection where code="%s"'''%car_name[1]
                new_mysql.csl.execute(sql)
                result = new_mysql.csl.fetchall()
                if not result:
                    print(car_name[1])
                    sql = self.carsInfo_sql(car_name[1], car_name, name_dict, i_name, new_mysql)
                    new_mysql.csl.execute(sql)
                    self._save_to_file('add_cars_info', sql)
                    self.sqlCount += 1
                    if self.sqlCount % 1000 == 0:
                        new_mysql.commit()
                else:
                    # print('开始抓取------------------')
                    continue
        new_mysql.commit()

    def stoPostData(self,new_mysql):
        sql = '''SELECT code FROM cars_info_collection_add WHERE `name` = "%s"'''%self.regnDict[self.regn]
        print(self.regnDict[self.regn])
        new_mysql.csl.execute(sql)
        result = new_mysql.csl.fetchall()
        for code in list(result):
            carinfo_url = self.carUrl.format(code[0])
            only_response = self.read_get_response(carinfo_url)
            # 获取get响应内容
            for mod in self.modis_list:
                data_list = self.parse_data(only_response)
                data = self.parse_postdata(data_list)
                self.PostData = data
                # # 构建post参数
                data['LRCD'] = mod
                # 构建sql语句
                sql_data = self.group_collection_sql(data, mod, new_mysql)
                sql_data = self.postData_sql(new_mysql,self.PostData['BPNO'] + self.PostData['LRCD'])
                self.sqlCount += 1
                if self.sqlCount % 1000 == 0:
                    new_mysql.commit()
        new_mysql.commit()

    def stoStrucData(self,new_mysql):
        sql = '''SELECT code FROM `cars_info_collection`'''
        new_mysql.csl.execute(sql)
        result = new_mysql.csl.fetchall()
        print(result)
        for code in result:
            print(code)
            sql = '''SELECT * FROM `%s` WHERE bpno="%s"'''%('Gpost_data_'+self.regn, code[0])
            new_mysql.dictCsl.execute(sql)
            result = new_mysql.dictCsl.fetchall()
            for data in result:
                # 构建子组post参数   分组编号参数
                print(data['GRNO'])
                zizu_url = self.Gurl.format(data['GRNO'])
                # 构建子组url
                postData = self.setPostdata(data)
                url_md5 = zizu_url + json.dumps(postData)
                html_name = hashlib.md5(url_md5.encode()).hexdigest()
                # 拼接加密
                response = self.read_post_response(html_name, zizu_url, data, HKGB=data['HKGB'], dirs=data['BPNO'] + '/' + data['LRCD'])
                # 获取子组响应
                rStrip = response.strip('\@')
                zizu_img = re.split(r'\\@', rStrip)[0]
                # print(zizu_img)
                img_item_list = self.zizu_parse_img(zizu_img)
                page_item_ = self.parse_item_list_(img_item_list)
                page_item = self.parse_item_list(img_item_list)
                zizu_list = re.split(r'\\@', response)[1].strip('\^')
                new_zizu = re.split(r'\\\^', zizu_list)
                zizu_img_num = re.split(r'\\@',response)[2].strip('\^')
                zizu_img_n = re.split(r'\\\^',zizu_img_num)
                StrData = dict()
                StrData['zizu_img_n'] = json.dumps(zizu_img_n)
                StrData['page_item'] = json.dumps(page_item)
                StrData['page_item_'] = json.dumps(page_item_)
                StrData['new_zizu'] = json.dumps(new_zizu)
                StrData['uid'] = data['uid']
                self.PostData = StrData
                self.StrucData(new_mysql)
                self.sqlCount += 1
                if self.sqlCount % 1000 == 0:
                    new_mysql.commit()
            new_mysql.commit()

    def stoGpostData(self,new_mysql):
        sql = '''select * from cars_info_collection_add'''
        new_mysql.dictCsl.execute(sql)
        cidlist = new_mysql.dictCsl.fetchall()
        for code in cidlist:
            print(code['code'])
            sql = '''SELECT * FROM `%s` where BPNO="%s"'''%('post_data_'+self.regn,code['code'])
            new_mysql.dictCsl.execute(sql)
            result = new_mysql.dictCsl.fetchall()
            for data in result:
                print(data)
                data['id'] = 0
                    # 分组url
                url_md5 = self.modisUrl + data['LRCD']
                # 拼接加密
                html_name = hashlib.md5(url_md5.encode()).hexdigest()
                fenzu_response = self.read_post_response(html_name=html_name, post_url=self.modisUrl, data=data, dirs=data['BPNO'] + '/' + data['LRCD'])
                # 获取分组响应
                fenzu_test = self.parse_fenzu(fenzu_response)
                # 解析出分组的内容
                page_set = self.perse_fenzu_img(fenzu_response)
                # 分组图片
                imageDict = self.strucImg(page_set,data,html_name)
                for i in fenzu_test:
                    if not i:
                        continue
                    new_i_list = re.split(r'\\:', i)
                    # 取出分组中的组件名称
                    data['GRNO'] = new_i_list[2]
                    data['strucName'] = new_i_list[1]
                    self.PostData = data
                    self.GpostData_sql(new_mysql)
                    self.sqlCount += 1
                    if self.sqlCount % 1000 == 0:
                        new_mysql.commit()
            new_mysql.commit()

    def stoStructure(self,new_mysql):
        sql = '''select cid from cars_group_collection'''
        new_mysql.csl.execute(sql)
        code_list = new_mysql.csl.fetchall()
        for code in code_list:
            print(code)
            sql = '''SELECT * FROM `%s` where uid LIKE "%%%s%%"''' % ('StrucData_' + self.regn,code[0])
            new_mysql.dictCsl.execute(sql)
            result = new_mysql.dictCsl.fetchall()
            for strdata in result:
                sql = '''SELECT * FROM `%s` WHERE uid = "%s"''' % ('Gpost_data_' + self.regn, strdata['uid'])
                new_mysql.dictCsl.execute(sql)
                data = new_mysql.dictCsl.fetchone()
                zizu_url = self.Gurl.format(data['GRNO'])
                url_md5 = zizu_url + data['uid']
                html_name = hashlib.md5(url_md5.encode()).hexdigest()
                zizu_img_n = json.loads(strdata['zizu_img_n'])
                page_item = json.loads(strdata['page_item'])
                for img_n in zizu_img_n:
                    img = re.split(r'\\:',img_n)
                    img_url_2 = 'http://wpc.mobis.co.kr/illust/GROUP/' + data['BPNO'] + '/TIF/' + data['GRNO'] + img[1] + img[2] + '.gif?time='+ str(int(time.time()))
                    img_name = data['BPNO'] + data['GRNO'] + img[1] + img[2]
                    print(img_url_2)
                    try:
                        img_response = self.read_img(data['BPNO'], img_name, HKGB=data['HKGB'])
                    except FileNotFoundError:
                        img_response = self.get_response(img_url_2, img=True)
                        print('图片不存在')
                        self.write_response(img_name, img_response, HKGB=data['HKGB'], dirs=data['BPNO'], img=True)
                    im_name = re.sub('-', '', data['GRNO']) + img[1]
                    if b'<!DOCTYPE' in img_response:
                        img_url_2 = re.sub(r'\*', '%5E', img_url_2)
                        img_response = self.get_response(img_url_2, img=True)
                        self.write_response(img_name, img_response, HKGB=data['HKGB'], dirs=data['BPNO'], img=True)
                        # print(img_response)
                    if b'<!DOCTYPE' in img_response:
                        continue
                    im_uid = hashlib.md5(img_response).hexdigest()
                    try:
                        sql = '''SELECT * FROM `image_hotspot_table` WHERE uid = "%s"''' % im_uid
                        new_mysql.csl.execute(sql)
                        result = new_mysql.csl.fetchone()
                    except Exception:
                        result = None
                    try:
                        im_hight,im_width = self.toPng(data['BPNO'], img_name, im_name=im_name, HKGB=data['HKGB'], result=result)
                    except Exception:
                        im_uid = hashlib.md5(img_response).hexdigest()
                        im_width = ''
                        im_hight = ''
                        print('图片错误')
                    try:
                        sql_data,imgPoints = self.structure_sql(page_item, img, im_hight, im_width, html_name, data, im_uid)
                        sql = new_mysql.insert_sql('cars_structure_collection',
                                               sql_data) + 'ON DUPLICATE KEY UPDATE `description`=values(`description`), `condition`=values(`condition`), `model`=values(`model`), `logo`=values(`logo`), `imgName`=values(`imgName`), `imageLarge`=values(`imageLarge`), `imgPoints`=values(`imgPoints`),`brandCode`=values(`brandCode`),`cid`=values(`cid`),`num`=values(`num`),`name`=values(`name`),`subgroup`=values(`subgroup`),`extra_info`=values(`extra_info`),`createtime`=values(`createtime`),`updatetime`=values(`updatetime`),`maingroup`=values(`maingroup`),`mid`=values(`mid`) '
                        new_mysql.csl.execute(sql)
                        # self._save_to_file('add_cars_structure_CHN', sql)
                        self.sqlCount += 1
                        if self.sqlCount % 1000 == 0:
                            new_mysql.commit()
                    except Exception as e:
                        print(e)
                        print('数据库异常')
                        continue
                    if result:
                        continue
                    sql_data = self.image_sql(im_uid, imgPoints, data, im_name)
                    try:
                        sql = new_mysql.insert_sql('image_hotspot_table',
                                                   sql_data) + 'ON DUPLICATE KEY UPDATE `imgPoints`=values(`imgPoints`),`brandCode`=values(`brandCode`),`createtime`=values(`createtime`),`imgPoints`=values(`imgPoints`),`updatetime`=values(`updatetime`),`status`=values(`status`) '
                        new_mysql.csl.execute(sql)
                        # self._save_to_file('add_image_hotspot_CHN', sql)
                        self.sqlCount += 1
                        if self.sqlCount % 1000 == 0:
                            new_mysql.commit()
                    except Exception:
                        print('数据库异常')
                new_mysql.commit()

    def stoDetail(self,new_mysql):
        sql = '''SELECT code FROM `cars_info_collection`'''
        new_mysql.csl.execute(sql)
        code_list = new_mysql.csl.fetchall()
        for code in code_list:
            print(code)
            sql = '''SELECT * FROM `%s` WHERE uid LIKE "%%%s%%"'''%('StrucData_' + self.regn, code[0])
            new_mysql.dictCsl.execute(sql)
            result = new_mysql.dictCsl.fetchall()
            for strdata in result:
                sql = '''SELECT * FROM `%s` WHERE uid = "%s"''' % ('Gpost_data_'+self.regn, strdata['uid'])
                new_mysql.dictCsl.execute(sql)
                data = new_mysql.dictCsl.fetchone()
                new_zizu = json.loads(strdata['new_zizu'])
                page_item_ = json.loads(strdata['page_item_'])
                page_num = page_item_.keys()
                for zizu in new_zizu:
                    new_zizu_list = re.split(r'\\:', zizu)
                    for w in page_num:
                        if '/' in new_zizu_list[0]:
                            new_zizu_list[0] = new_zizu_list[0].split('/')[1]
                        if new_zizu_list[0] in page_item_[w]:
                            if new_zizu_list[2] == 'MULTI':
                                new_zizu_list[2] = ''
                            # print(itemer)
                            postdata = self.setPostdata(data)
                            postdata['PNCD'] = new_zizu_list[0]
                            postdata['PTNO'] = new_zizu_list[2]
                            self.pncd = postdata['PNCD']
                            try:
                                if '/' in postdata['PNCD']:
                                    postdata['PNCD'] = re.findall(r'/(.*)', postdata['PNCD'])[0]
                            except Exception:
                                print('发生错误')
                                continue
                            # 构建详情post参数
                            detail_url = self.detailUrl.format(postdata['GRNO'], postdata['PNCD'])
                            url_md5 = detail_url + json.dumps(postdata)
                            html_name = hashlib.md5(url_md5.encode()).hexdigest()
                            response = self.read_post_response(html_name, detail_url, postdata, HKGB=postdata['HKGB'],dirs=postdata['BPNO'] + '/' + postdata['LRCD'])
                            rStrip = response.strip('@')
                            detail_list = re.split('@', rStrip)
                            # 分割成列表
                            for i in detail_list:
                                detail_data = re.split(r'\\:', i)
                                # print(detail_data)
                                # <---------------------sql-------------------->
                                try:
                                    sql_data = self.carsPartNum_sql(w ,postdata, detail_data)
                                    sql_data['uid'] = hashlib.md5(json.dumps(sql_data).encode()).hexdigest()
                                    sql_data['psort'] = self.psort
                                except:
                                    with open('bug.txt','a')as f:
                                        f.write(json.dumps(detail_data))
                                    print(self.sqlCount)
                                    continue
                                self.sqlCount += 1
                                sql = new_mysql.insert_sql('cars_part_num',sql_data) + 'ON DUPLICATE KEY UPDATE `brandCode`=values(`brandCode`)'
                                new_mysql.csl.execute(sql)
                                self._save_to_file('add_cars_part_GEN', sql)
                                if self.sqlCount % 1000 == 0:
                                    new_mysql.commit()
                                sql_data = self.partsPrice_sql(detail_data)
                                sql = new_mysql.insert_sql('parts_price_info', sql_data)+ 'ON DUPLICATE KEY UPDATE `brandCode`=values(`brandCode`),`name`=values(`name`),`condition`=values(`condition`),`qty`=values(`qty`),`label`=values(`label`),`createtime`=values(`createtime`),`channel`=values(`channel`),`retailprice`=values(`retailprice`),`status`=values(`status`) '
                                new_mysql.csl.execute(sql)
                                self._save_to_file('add_parts_price_GEN_old', sql)
                                if self.count % 1000 == 0:
                                    new_mysql.commit()
                            new_mysql.commit()

    def _get_file_fd(self, filename, append=False):
        if not hasattr(self, '_file_dict'):
            self._file_dict = {}
        if filename not in self._file_dict:
            m = 'a' if append else 'w'
            self._file_dict[filename] = open(filename, m)
        return self._file_dict[filename]

    def _save_to_file(self, filename, sql, append=False):
        fd = self._get_file_fd(filename, append)
        r = sql.strip().strip(';')
        fd.write(r+';\n')
        fd.flush()

    def run(self):
        new_mysql = Mysql(self.brandCode)
        # new_mysql = Mysql('hy')
        # old_mysql = Mysql('hy_old')
        old_mysql = Mysql('kia')
        # 车型表
        self.stoCarInfo(new_mysql, old_mysql)
        # self.stoPostData(new_mysql)
        # self.stoGpostData(new_mysql)
        # self.stoStrucData(new_mysql)
        # 分组表
        # self.stoStructure(new_mysql)
        # 零件详情表
        self.stoDetail(new_mysql)


if __name__ == '__main__':
    # hyundai  kia
    # CHN 国内，GEN 国外
    hyundai = Hyundai('kia', 'CHN')
    hyundai.run()